## creates tables for appendix B:
## sensitivity to measurement decisions
## tables A2, A3, A4, A5

rm(list=ls())

library(tidyverse)
library(kableExtra)


# table A2 - opinion gaps varying income threshold

opinion_thresholds <- readRDS('bootstrap_gaps_data/opinion_bootstrapped_classthresholds.RDS')

issuetopics <- opinion_thresholds$issuetopics

thresholds_out <- list()

for (party in c('dem', 'rep', 'all')) {
  for (thresh in c(5, 10, 20, 25, 33)) {
    tmp.top <- opinion_thresholds[[paste0(party, '_inctop_', thresh)]]
    tmp.bottom <- opinion_thresholds[[paste0(party, '_incbottom_', thresh)]]
    
    thresholds_out[[paste(party, thresh)]] <- data.frame(question = issuetopics$question,
                                                         gap_est = rowMeans(tmp.top - tmp.bottom),
                                                         gap_se = apply(tmp.top - tmp.bottom, MARGIN = 1, sd),
                                                         gap_sig = as.numeric(!apply(apply(tmp.top - tmp.bottom, MARGIN = 1, function(x) quantile(x, probs = c(0.025, 0.975))),
                                                                                     MARGIN = 2, function(y) between(0, y[1], y[2]))),
                                                         top_est = rowMeans(tmp.top),
                                                         top_se = apply(tmp.top, MARGIN = 1, sd),
                                                         bottom_est = rowMeans(tmp.bottom),
                                                         bottom_se = apply(tmp.bottom, MARGIN = 1, sd),
                                                         party = party,
                                                         threshold = thresh)
  }
}

thresholds_df <- bind_rows(thresholds_out) %>% 
  left_join(issuetopics %>% select(question, topic_6))

threshhold_results <- thresholds_df %>% 
  select(question, party, topic_6, gap_est, gap_sig, threshold) %>% 
  mutate(gap_est = abs(gap_est)) %>% 
  pivot_wider(names_from = threshold, values_from = c(gap_est, gap_sig)) %>% 
  group_by(party, topic_6) %>% 
  summarize_at(vars(starts_with('gap_')), mean) %>% 
  ungroup() %>% 
  bind_rows(thresholds_df %>% 
              select(question, party, gap_est, gap_sig, threshold) %>% 
              mutate(gap_est = abs(gap_est)) %>% 
              pivot_wider(names_from = threshold, values_from = c(gap_est, gap_sig)) %>% 
              group_by(party) %>% 
              summarize_at(vars(starts_with('gap_')), mean) %>% 
              ungroup() %>% 
              mutate(topic_6 = 'all_issues'))

threshhold_results %>% 
  select(party, topic_6, starts_with('gap_est_')) %>% 
  arrange(party) %>% 
  mutate(party = case_when(party == 'rep' ~ 'Republicans',
                           party == 'dem' ~ 'Democrats',
                           party == 'all' ~ 'All Respondents'),
         topic_6 = case_when(topic_6 == 'foreignpolicy' ~ 'Foreign policy',
                             topic_6 == 'cultural' ~ 'Cultural',
                             topic_6 == 'immigration' ~ 'Immigration',
                             topic_6 == 'lawenforcement' ~ 'Law enforcement',
                             topic_6 == 'economic' ~ 'Economic',
                             topic_6 == 'socialwelfare' ~ 'Social welfare',
                             topic_6 == 'all_issues' ~ 'All Issues')) %>% 
  mutate(party = cell_spec(party, angle = 90, format = 'latex')) %>%
  mutate_if(is.numeric, round, 3) %>% 
  kable(booktabs = TRUE, 'latex', escape = FALSE, linesep = c(rep('', 6), '\\addlinespace'),
        col.names = linebreak(c(' ', 'Policy\nDomain', '5', '10', '20', '25', '33')),
        align = c('l', 'l', rep('c', 5))) %>% 
  collapse_rows(1, latex_hline = 'major') %>% 
  row_spec(c(7, 14, 21), bold = TRUE) %>% 
  add_header_above(c(' ' = 2, 'Percentile Thresholds for Income Groups' = 5)) %>%
  save_kable('appendix_tables/thresholds_results.tex')


# table A3 - results with partisan leaners

rm(list=ls())

opinion_bootstrapped <- readRDS('bootstrap_gaps_data/opinion_bootstrapped_thresholdwithleaners.RDS')

for (i in c('issuetopics',
            names(opinion_bootstrapped)[grepl('_10$', names(opinion_bootstrapped))])) {
  assign(i, opinion_bootstrapped[[i]])
}

dem_polarization_boot <- dem_inctop_10 - dem_incbottom_10
rep_polarization_boot <- rep_inctop_10 - rep_incbottom_10

polarization_tab <- data.frame(expand.grid(party = c('rep', 'dem'),
                                           topic_6 = unique(issuetopics$topic_6),
                                           stringsAsFactors = FALSE),
                               avgpol = NA,
                               signif = NA,
                               gilens = NA,
                               gilenssignif = NA,
                               disagree = NA,
                               disagreegilens = NA,
                               n_quest = NA,
                               stringsAsFactors = FALSE)

for (i in 1:nrow(polarization_tab)) {
  temp.issues <- issuetopics$question[issuetopics$topic_6 == polarization_tab$topic_6[i]]
  temp.rich <- get(paste0(polarization_tab$party[i], '_inctop_10'))
  temp.rich <- temp.rich[rownames(temp.rich) %in% temp.issues,]
  temp.poor <- get(paste0(polarization_tab$party[i], '_incbottom_10'))
  temp.poor <- temp.poor[rownames(temp.poor) %in% temp.issues,]
  temp.pol <- get(paste0(polarization_tab$party[i], '_polarization_boot'))
  temp.pol <- temp.pol[rownames(temp.pol) %in% temp.issues,]
  
  polarization_tab$avgpol[i] <- median(colMeans(abs(temp.pol)))
  
  temp.ci <- apply(temp.rich - temp.poor, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab$signif[i] <- sum(!apply(temp.ci, MARGIN = 2, function(x) between(0, x[1], x[2])))
  
  polarization_tab$gilens[i] <- sum(abs(apply(temp.pol, MARGIN = 1, median)) > 0.1)
  
  temp.ci <- apply(temp.pol, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab$gilenssignif[i] <- sum(apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1)))
  
  temp.10pt <- apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1))
  
  temp.disagree <- abs(apply((temp.rich > 0.5) - (temp.poor > 0.5), MARGIN =  1, median))
  
  polarization_tab$disagree[i] <- sum(temp.disagree)
  
  polarization_tab$disagreegilens[i] <- sum(temp.disagree*as.numeric(temp.10pt))
  
  polarization_tab$n_quest[i] <- length(temp.issues)
}

for (p in unique(polarization_tab$party)) {
  i <- nrow(polarization_tab) + 1
  polarization_tab[i,] <- NA
  polarization_tab$topic_6[i] <- 'Total'
  polarization_tab$party[i] <- p
  
  temp.pol <- get(paste0(polarization_tab$party[i], '_polarization_boot'))
  
  polarization_tab$avgpol[i] <- median(colMeans(abs(temp.pol)))
  
  for (v in c('signif', 'gilens', 'gilenssignif', 'disagree', 'disagreegilens', 'n_quest')) { 
    polarization_tab[i,v] <- sum(polarization_tab[polarization_tab$party == p, v], na.rm = TRUE)
  }
}

polarization_tab %>% 
  select(party, topic_6, n_quest, avgpol, signif, gilens, disagreegilens) %>% 
  arrange(party) %>%
  mutate(party = case_when(party == 'rep' ~ 'Republicans',
                           party == 'dem' ~ 'Democrats'),
         topic_6 = case_when(topic_6 == 'foreignpolicy' ~ 'Foreign policy',
                             topic_6 == 'cultural' ~ 'Cultural',
                             topic_6 == 'immigration' ~ 'Immigration',
                             topic_6 == 'lawenforcement' ~ 'Law enforcement',
                             topic_6 == 'economic' ~ 'Economic',
                             topic_6 == 'socialwelfare' ~ 'Social welfare',
                             TRUE ~ topic_6)) %>% 
  mutate_at(vars(signif:disagreegilens), funs(. / n_quest)) %>% 
  mutate_if(is.numeric, round, 3) %>% 
  mutate_at(vars(signif:disagreegilens), funs(paste0(. * 100, '\\%'))) %>% 
  mutate(party = cell_spec(party, angle = 90, format = 'latex')) %>% 
  kable('latex', booktabs = TRUE, escape = FALSE, linesep = '\\addlinespace',
        col.names = linebreak(c(' ', 'Policy\nDomain', 'Number of\nUnique Issues', 'Average\nOpinion Gap', 'Opinion Difference\nStatistically\nSignificant', 
                                'Opinion Gap\nGreater than\n10 pts.', '10 pt. Gap +\nDisagreement\non Outcome'),
                              align = 'c'),
        align = c('l', 'l', rep('c', ncol(.)-2))) %>%
  collapse_rows(1, latex_hline = 'major') %>% 
  row_spec(c(7, 14), bold = TRUE) %>% 
  save_kable(file = 'appendix_tables/detailed_polarization_withleaners.tex')



# table A4 - define income groups within party

rm(list = ls())

opinion_bootstrapped <- readRDS('bootstrap_gaps_data/opinion_bootstrapped_withinparty.RDS')

for (i in names(opinion_bootstrapped)) {
  assign(i, opinion_bootstrapped[[i]])
}

dem_polarization_boot <- dem_inctop_10 - dem_incbottom_10
rep_polarization_boot <- rep_inctop_10 - rep_incbottom_10

polarization_tab <- data.frame(expand.grid(party = c('rep', 'dem'),
                                           topic_6 = unique(issuetopics$topic_6),
                                           stringsAsFactors = FALSE),
                               avgpol = NA,
                               signif = NA,
                               gilens = NA,
                               gilenssignif = NA,
                               disagree = NA,
                               disagreegilens = NA,
                               n_quest = NA,
                               stringsAsFactors = FALSE)

for (i in 1:nrow(polarization_tab)) {
  temp.issues <- issuetopics$question[issuetopics$topic_6 == polarization_tab$topic_6[i]]
  temp.rich <- get(paste0(polarization_tab$party[i], '_inctop_10'))
  temp.rich <- temp.rich[rownames(temp.rich) %in% temp.issues,]
  temp.poor <- get(paste0(polarization_tab$party[i], '_incbottom_10'))
  temp.poor <- temp.poor[rownames(temp.poor) %in% temp.issues,]
  temp.pol <- get(paste0(polarization_tab$party[i], '_polarization_boot'))
  temp.pol <- temp.pol[rownames(temp.pol) %in% temp.issues,]
  
  polarization_tab$avgpol[i] <- median(colMeans(abs(temp.pol)))
  
  temp.ci <- apply(temp.rich - temp.poor, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab$signif[i] <- sum(!apply(temp.ci, MARGIN = 2, function(x) between(0, x[1], x[2])))
  
  polarization_tab$gilens[i] <- sum(abs(apply(temp.pol, MARGIN = 1, median)) > 0.1)
  
  temp.ci <- apply(temp.pol, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab$gilenssignif[i] <- sum(apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1)))
  
  temp.10pt <- apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1))
  
  temp.disagree <- abs(apply((temp.rich > 0.5) - (temp.poor > 0.5), MARGIN =  1, median))
  
  polarization_tab$disagree[i] <- sum(temp.disagree)
  
  polarization_tab$disagreegilens[i] <- sum(temp.disagree*as.numeric(temp.10pt))
  
  polarization_tab$n_quest[i] <- length(temp.issues)
}

for (p in unique(polarization_tab$party)) {
  i <- nrow(polarization_tab) + 1
  polarization_tab[i,] <- NA
  polarization_tab$topic_6[i] <- 'Total'
  polarization_tab$party[i] <- p
  
  temp.pol <- get(paste0(polarization_tab$party[i], '_polarization_boot'))
  
  polarization_tab$avgpol[i] <- median(colMeans(abs(temp.pol)))
  
  for (v in c('signif', 'gilens', 'gilenssignif', 'disagree', 'disagreegilens', 'n_quest')) { 
    polarization_tab[i,v] <- sum(polarization_tab[polarization_tab$party == p, v], na.rm = TRUE)
  }
}

polarization_tab %>% 
  select(party, topic_6, n_quest, avgpol, signif, gilens, disagreegilens) %>% 
  arrange(party) %>%
  mutate(party = case_when(party == 'rep' ~ 'Republicans',
                           party == 'dem' ~ 'Democrats'),
         topic_6 = case_when(topic_6 == 'foreignpolicy' ~ 'Foreign policy',
                             topic_6 == 'cultural' ~ 'Cultural',
                             topic_6 == 'immigration' ~ 'Immigration',
                             topic_6 == 'lawenforcement' ~ 'Law enforcement',
                             topic_6 == 'economic' ~ 'Economic',
                             topic_6 == 'socialwelfare' ~ 'Social welfare',
                             TRUE ~ topic_6)) %>% 
  mutate_at(vars(signif:disagreegilens), funs(. / n_quest)) %>% 
  mutate_if(is.numeric, round, 3) %>% 
  mutate_at(vars(signif:disagreegilens), funs(paste0(. * 100, '\\%'))) %>% 
  mutate(party = cell_spec(party, angle = 90, format = 'latex')) %>% 
  kable('latex', booktabs = TRUE, escape = FALSE, linesep = '\\addlinespace',
        col.names = linebreak(c(' ', 'Policy\nDomain', 'Number of\nUnique Issues', 'Average\nOpinion Gap', 'Opinion Difference\nStatistically\nSignificant', 
                                'Opinion Gap\nGreater than\n10 pts.', '10 pt. Gap +\nDisagreement\non Outcome'),
                              align = 'c'),
        align = c('l', 'l', rep('c', ncol(.)-2))) %>%
  collapse_rows(1, latex_hline = 'major') %>% 
  row_spec(c(7, 14), bold = TRUE) %>% 
  save_kable(file = 'appendix_tables/detailed_polarization_within_party.tex')


# table A5 - do not drop issues in early years

rm(list=ls())

opinion_bootstrapped <- readRDS('bootstrap_gaps_data/opinion_bootstrapped_allquestions.RDS')

for (i in names(opinion_bootstrapped)) {
  assign(i, opinion_bootstrapped[[i]])
}

dem_polarization_boot <- dem_top_boot - dem_bottom_boot
rep_polarization_boot <- rep_top_boot - rep_bottom_boot
all_polarization_boot <- all_top_boot - all_bottom_boot

polarization_tab <- data.frame(expand.grid(party = c('rep', 'dem', 'all'),
                                           topic_6 = unique(issuetopics$topic_6),
                                           stringsAsFactors = FALSE),
                               avgpol = NA,
                               signif = NA,
                               gilens = NA,
                               gilenssignif = NA,
                               disagree = NA,
                               disagreegilens = NA,
                               n_quest = NA,
                               stringsAsFactors = FALSE)

for (i in 1:nrow(polarization_tab)) {
  temp.issues <- unique(issuetopics$question[issuetopics$topic_6 == polarization_tab$topic_6[i]])
  temp.rich <- get(paste0(polarization_tab$party[i], '_top_boot'))
  temp.rich <- temp.rich[issuetopics$question %in% temp.issues,]
  temp.poor <- get(paste0(polarization_tab$party[i], '_bottom_boot'))
  temp.poor <- temp.poor[issuetopics$question %in% temp.issues,]
  temp.pol <- get(paste0(polarization_tab$party[i], '_polarization_boot'))
  temp.pol <- temp.pol[issuetopics$question %in% temp.issues,]
  
  polarization_tab$avgpol[i] <- median(colMeans(abs(temp.pol)))
  
  temp.ci <- apply(temp.rich - temp.poor, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab$signif[i] <- sum(!apply(temp.ci, MARGIN = 2, function(x) between(0, x[1], x[2])))
  
  polarization_tab$gilens[i] <- sum(abs(apply(temp.pol, MARGIN = 1, median)) > 0.1)
  
  temp.ci <- apply(temp.pol, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab$gilenssignif[i] <- sum(apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1)))
  
  temp.10pt <- apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1))
  
  temp.disagree <- abs(apply((temp.rich > 0.5) - (temp.poor > 0.5), MARGIN =  1, median))
  
  polarization_tab$disagree[i] <- sum(temp.disagree)
  
  polarization_tab$disagreegilens[i] <- sum(temp.disagree*as.numeric(temp.10pt))
  
  polarization_tab$n_quest[i] <- nrow(temp.pol)
}

for (p in unique(polarization_tab$party)) {
  i <- nrow(polarization_tab) + 1
  polarization_tab[i,] <- NA
  polarization_tab$topic_6[i] <- 'Total'
  polarization_tab$party[i] <- p
  
  temp.pol <- get(paste0(polarization_tab$party[i], '_polarization_boot'))
  
  polarization_tab$avgpol[i] <- median(colMeans(abs(temp.pol)))
  
  for (v in c('signif', 'gilens', 'gilenssignif', 'disagree', 'disagreegilens', 'n_quest')) { 
    polarization_tab[i,v] <- sum(polarization_tab[polarization_tab$party == p, v], na.rm = TRUE)
  }
}

polarization_tab %>% 
  select(party, topic_6, n_quest, avgpol, signif, gilens, disagreegilens) %>% 
  arrange(party) %>%
  mutate(party = case_when(party == 'rep' ~ 'Republicans',
                           party == 'dem' ~ 'Democrats',
                           party == 'all' ~ 'All Respondents'),
         topic_6 = case_when(topic_6 == 'foreignpolicy' ~ 'Foreign policy',
                             topic_6 == 'cultural' ~ 'Cultural',
                             topic_6 == 'immigration' ~ 'Immigration',
                             topic_6 == 'lawenforcement' ~ 'Law enforcement',
                             topic_6 == 'economic' ~ 'Economic',
                             topic_6 == 'socialwelfare' ~ 'Social welfare',
                             TRUE ~ topic_6)) %>% 
  mutate_at(vars(signif:disagreegilens), funs(. / n_quest)) %>% 
  mutate_if(is.numeric, round, 3) %>% 
  mutate_at(vars(signif:disagreegilens), funs(paste0(. * 100, '\\%'))) %>% 
  mutate(party = cell_spec(party, angle = 90, format = 'latex')) %>% 
  kable('latex', booktabs = TRUE, escape = FALSE, linesep = '\\addlinespace',
        col.names = linebreak(c(' ', 'Policy\nDomain', 'Number of\nQuestions', 'Average\nOpinion Gap', 'Opinion Difference\nStatistically\nSignificant', 
                                'Opinion Gap\nGreater than\n10 pts.', '10 pt. Gap+\nDisagreement\non Outcome'),
                              align = 'c'),
        align = c('l', 'l', rep('c', ncol(.)-2))) %>%
  collapse_rows(1, latex_hline = 'major') %>% 
  row_spec(c(7, 14, 21), bold = TRUE) %>% 
  save_kable(file = 'appendix_tables/detailed_polarization_allissues_nodrop.tex')
